﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Web;
using System.IO;
using System.Net;
using System.Text.RegularExpressions;
using com.crawler.common;

namespace com.crawler.components
{
    public class NetSiteCatchManager
    {
        /// <summary>
        /// 页面超时时间
        /// </summary>
        const int c_TIME_OUT = 20000;

        /// <summary>
        /// 锁对象，防止线程并发
        /// </summary>
        static object m_ojb = new object();
        /// <summary>
        /// 读取页面内容
        /// </summary>
        /// <param name="i_url"></param>
        /// <param name="i_encoding"></param>
        /// <returns></returns>
        public static string ReadUrl(string i_url, Encoding i_encoding)
        {


            Encoding t_encoding = null;
            string t_html = string.Empty;
            try
            {
                using (MemoryStream t_memory_stream = new MemoryStream())
                {
                    ReadPageCodeToStream(i_url, t_memory_stream, delegate(HttpWebResponse t_response)
                    {
                        t_encoding = i_encoding ?? GetEncoding(t_response);
                    });

                    t_memory_stream.Position = 0;//内存流复位以便于重新读取

                    using (StreamReader t_reader = new StreamReader(t_memory_stream, t_encoding))
                    {
                        t_html = t_reader.ReadToEnd();
                    }
                }
                if (!string.IsNullOrEmpty(t_html))
                {
                    t_html = t_html.Replace("\r", "").Replace("\n", "").Replace("\r\n", "");
                }
                
                return t_html;
            }
            catch (Exception ex)
            {

                LogHelper.Error("获取页面内容失败" + ex.ToString() + ",url=" + i_url);
                ex = null;
                return string.Empty;
            }
        }

        /// <summary>
        /// 获取页面编码
        /// </summary>
        /// <param name="response"></param>
        /// <returns></returns>
        private static Encoding GetEncoding(HttpWebResponse response)
        {

            Encoding t_encoding = Encoding.Default;
            if (response.CharacterSet != null)
            {
                try
                {
                    t_encoding = Encoding.GetEncoding(response.CharacterSet);
                }
                catch (ArgumentException ex)
                {
                    LogHelper.Error("获取页面编码失败" + ex.ToString());
                }
            }
            return t_encoding;
        }

        /// <summary>
        /// 读取页面内容到数据流
        /// </summary>
        /// <param name="i_url"></param>
        /// <param name="i_stream"></param>
        /// <param name="ResponseCallback"></param>
        private static void ReadPageCodeToStream(string i_url, Stream i_stream, Action<HttpWebResponse> ResponseCallback)
        {
            HttpWebRequest t_request = (HttpWebRequest)WebRequest.Create(i_url);
            t_request.Timeout = c_TIME_OUT;
            t_request.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.5) Gecko/2008120122 Firefox/3.0.5 (.NET CLR";
            t_request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
            t_request.Headers["Accept-Language"] = "en-us,en;q=0.5";
            t_request.Headers["Accept-Charset"] = "ISO-8859-1,utf-8;q=0.7,*;q=0.7";

            using (HttpWebResponse t_response = (HttpWebResponse)t_request.GetResponse())
            {
                if (ResponseCallback != null)
                {
                    ResponseCallback(t_response);
                }

                Stream t_stream = t_response.GetResponseStream();

                byte[] t_buffer = new byte[10000];
                int t_count = 0;
                while ((t_count = t_stream.Read(t_buffer, 0, t_buffer.Length)) != 0)
                {
                    i_stream.Write(t_buffer, 0, t_count);
                }
            }
        }

        /// <summary>
        /// 过滤字符
        /// </summary>
        /// <param name="i_text"></param>
        /// <returns></returns>
        public static string ReplaceContent(string i_text)
        {
            if (string.IsNullOrEmpty(i_text))
                return string.Empty;
            i_text = Regex.Replace(i_text, "<form[^<>]*?>.*?</form>", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            MatchCollection t_ma = Regex.Matches(i_text, "<img[^<>]*?>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            if (t_ma != null && t_ma.Count > 8)
            {
                i_text = Regex.Replace(i_text, "<img[^<>]*?>", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            }
           
            i_text = Regex.Replace(i_text, "<[/]*(?=div|table|td)[^<>]*?>", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            i_text = Regex.Replace(i_text, "<!--.*?-->", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            i_text = Regex.Replace(i_text, "<a.*?>.*?</a>", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            i_text = Regex.Replace(i_text, "<script.*?>.*?</script>", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            i_text = Regex.Replace(i_text, "<[/]*?iframe.*?>", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            i_text = Regex.Replace(i_text, "<font[^<>]*>.*?</font>", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            i_text = Regex.Replace(i_text, "{.*?}", "", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            i_text = Regex.Replace(i_text, "<b[^<>]*>.*?</b>", "", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            i_text = Regex.Replace(i_text, "<ul[^<>]*>.*?</ul>", "", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            i_text = Regex.Replace(i_text, "<li[^<>]*>.*?</li>", "", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            i_text = i_text.Replace("起点中文网  欢迎广大书友光临阅读，最新、最快、最火的连载作品尽在起点原创！", "");
            i_text = i_text.Replace("起点中文网欢迎广大书友光临阅读，最新、最快、最火的连载作品尽在起点原创！", "");
            i_text = i_text.Replace("起点中文网", "");
            i_text = i_text.Replace("尽在起点原创！", "");
            return i_text;
        }

        /// <summary>
        /// 确定是否是章节内容
        /// </summary>
        /// <param name="i_book_name"></param>
        /// <param name="i_chapter_name"></param>
        /// <param name="i_html"></param>
        /// <returns></returns>
        public static bool IsContainChapterName(string i_book_name, string i_chapter_name, string i_html)
        {

            //去掉空格

            i_book_name = Regex.Replace(i_book_name, "[【】（），！？(),!?;\\.；、……]", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);

            i_chapter_name = Regex.Replace(i_chapter_name, "[\\s【】（），！？(),!?;；\\.、……]", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);

            i_html = Regex.Replace(i_html, "[\\s【】（），！？(),!?;；\\.、……]", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            if (i_html.IndexOf(i_chapter_name) > -1 && i_html.IndexOf(i_book_name) > -1)
                return true;

            return false;
        }

        /// <summary>
        /// 判断是否是相同的章节
        /// </summary>
        /// <param name="i_chapter_source"></param>
        /// <param name="i_chapter_target"></param>
        /// <returns></returns>
        public static bool CompareChapterName(string i_chapter_source, string i_chapter_target)
        {
            if (string.IsNullOrEmpty(i_chapter_source) || string.IsNullOrEmpty(i_chapter_target))
                return false;

            if (i_chapter_source.Equals(i_chapter_target))
                return true;

            //去掉空格
            i_chapter_source = Regex.Replace(i_chapter_source, "\\s", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            i_chapter_source = Regex.Replace(i_chapter_source, "[【】（）a-zA-Z，！？(),!?;\\.；、……]", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            i_chapter_target = Regex.Replace(i_chapter_target, "\\s", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
            i_chapter_target = Regex.Replace(i_chapter_target, "[【】（）a-zA-Z，！？(),!?;；\\.、……]", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);

            if (string.IsNullOrEmpty(i_chapter_source) || string.IsNullOrEmpty(i_chapter_target))
                return false;

            if (i_chapter_source.IndexOf(i_chapter_target) > -1 || i_chapter_target.IndexOf(i_chapter_source) > -1)
                return true;

            return false;
        }

        /// <summary>
        /// 写书的章节文本
        /// </summary>
        /// <param name="i_book_id"></param>
        /// <param name="i_chapter_id"></param>
        /// <param name="i_text"></param>
        /// <returns></returns>
        public static bool WriteBookContent(string i_chapter_path, long i_book_id, long i_chapter_id, string i_text)
        {
   
            i_text = string.Format("document.write('{0}');", i_text);
            lock (m_ojb)
            {
                string t_path = i_chapter_path;
                t_path = Path.Combine(t_path, i_book_id.ToString());
                t_path = Path.Combine(t_path, "txt");
                if (!Directory.Exists(t_path))
                {
                    Directory.CreateDirectory(t_path);
                }

                string t_file_name = Path.Combine(t_path, string.Format("{0}.txt", i_chapter_id));
                try
                {
                    using (FileStream t_file_stream = new FileStream(t_file_name, FileMode.Append, FileAccess.Write))
                    {
                        using (StreamWriter t_writer = new StreamWriter(t_file_stream, Encoding.Unicode))
                        {
                            t_writer.WriteLine(i_text);
                            t_writer.Flush();
                            t_writer.Close();
                        }
                    }
                    return true;
                }
                catch (Exception ex)
                {
                    LogHelper.Error("写文件信息失败" + ex.ToString());
                    return false;
                }
            }
        }

        /// <summary>
        /// 下载文件
        /// </summary>
        /// <param name="i_img_url"></param>
        /// <param name="i_file_name"></param>
        public static string DownLoadFile(string i_url, string i_chapter_path, long i_book_id, long i_chapter_id)
        {

            string t_path = i_chapter_path;
            t_path = Path.Combine(t_path, i_book_id.ToString());
            t_path = Path.Combine(t_path, "pic");
            t_path= Path.Combine(t_path, i_chapter_id.ToString());
            if (!Directory.Exists(t_path))
            {
                Directory.CreateDirectory(t_path);
            }


            string t_url_file_name = string.Format("{0}.gif", Guid.NewGuid().ToString());
            string t_file_name = Path.Combine(t_path, t_url_file_name);
            try
            {
                Uri u = new Uri(i_url);
                System.Net.HttpWebRequest mRequest = (HttpWebRequest)WebRequest.Create(u);
                mRequest.Method = "GET";
                mRequest.ContentType = "Application/octet-stream";
                HttpWebResponse wr = (HttpWebResponse)mRequest.GetResponse();
                long length = wr.ContentLength;
                if (length < 100)
                    return "noimg";

                Stream sIn = wr.GetResponseStream();
                FileStream fs = new FileStream(t_file_name, FileMode.Create, FileAccess.Write);
                
                int i = 0;
                long j = 0;

                byte[] buffer = new byte[1024];
                while ((i = sIn.Read(buffer, 0, buffer.Length)) > 0)
                {
                    j += i;
                    fs.Write(buffer, 0, i);

                }

                sIn.Close();
                wr.Close();

                fs.Close();

                return t_url_file_name;

            }
            catch(Exception ex)
            {
                LogHelper.Error("下载图片失败" + ex.ToString());
                return string.Empty;
            }

        }

        /// <summary>
        /// 下载小说封面
        /// </summary>
        /// <param name="i_icon_url"></param>
        /// <param name="i_book_id"></param>
        /// <param name="i_icon_path"></param>
        public static void DownBookICON(string i_icon_url, long i_book_id, string i_icon_path)
        {
            
            if (!Directory.Exists(i_icon_path))
            {
                Directory.CreateDirectory(i_icon_path);
            }

            string t_file_name = Path.Combine(i_icon_path, string.Format("{0}.jpg", i_book_id));
            try
            {
                Uri u = new Uri(i_icon_url);
                System.Net.HttpWebRequest mRequest = (HttpWebRequest)WebRequest.Create(u);
                mRequest.Method = "GET";
                mRequest.ContentType = "Application/octet-stream";
                HttpWebResponse wr = (HttpWebResponse)mRequest.GetResponse();
                long length = wr.ContentLength;
  

                Stream sIn = wr.GetResponseStream();
                FileStream fs = new FileStream(t_file_name, FileMode.Create, FileAccess.Write);
               
                int i = 0;
                long j = 0;

                byte[] buffer = new byte[1024];
                while ((i = sIn.Read(buffer, 0, buffer.Length)) > 0)
                {
                    j += i;
                    fs.Write(buffer, 0, i);

                }

                sIn.Close();
                wr.Close();

                fs.Close();

               
            }
            catch (Exception ex)
            {
                LogHelper.Error("下载图片失败" + ex.ToString());
          
               
            }
        }


        /// <summary>
        /// 获取站点完整链接地址
        /// </summary>
        /// <param name="i_site_url"></param>
        /// <param name="i_part_url"></param>
        /// <returns></returns>
        public static string GetFullUrl(string i_site_url, string i_part_url)
        {
            if (i_site_url.IndexOf("#") > -1 || i_part_url.IndexOf("#") > -1)
                return string.Empty;

            if(i_part_url.IndexOf("http://")>-1)
                return i_part_url;

          

            if (!i_part_url.StartsWith("http") && !i_part_url.StartsWith("/"))
            {
                i_part_url = i_site_url.Substring(0, i_site_url.LastIndexOf("/") + 1) + i_part_url;
            }
           
            if (i_part_url.StartsWith("/"))
            {
                Uri t_url = new Uri(i_site_url);

                i_part_url ="http://"+ t_url.Host+ i_part_url;
            }
            return i_part_url;

        }

        /// <summary>
        /// 是否是盗版网站
        /// </summary>
        /// <param name="i_url"></param>
        /// <returns></returns>
        public static bool IsPiraticSite(string i_url)
        {
            if (string.IsNullOrEmpty(i_url))
                return false;

            if (!Regex.IsMatch(i_url, "http://[^/]+/\\S+", RegexOptions.IgnoreCase))
                return false;

            if (i_url.ToLower().IndexOf("tieba") > -1 || i_url.ToLower().IndexOf("bbs") > -1 || i_url.ToLower().IndexOf("vip") > -1)
                return false;

            if (i_url.ToLower().IndexOf("tiyma.com") > -1)
                return false;

            if (i_url.ToLower().IndexOf("qidian.com") > -1)
                return false;

            if (i_url.ToLower().IndexOf("readnovel.com") > -1)
                return false;

            if (i_url.ToLower().IndexOf("hongxiu.com") > -1)
                return false;

            if (i_url.ToLower().IndexOf("xxsy.net") > -1)
                return false;

            if (i_url.ToLower().IndexOf("jjwxc.net") > -1)
                return false;

            if (i_url.ToLower().IndexOf("booksky.org") > -1)
                return false;

            if (i_url.ToLower().IndexOf("sina.com.cn") > -1)
                return false;

            if (i_url.ToLower().IndexOf("zhengheng.com") > -1)
                return false;

            if (i_url.ToLower().IndexOf("zengheng.com") > -1)
                return false;

            if (i_url.ToLower().IndexOf("kuaikanshu.com") > -1)
                return false;

            if (i_url.ToLower().IndexOf("yunxuange.com") > -1)
                return false;

            if (i_url.ToLower().IndexOf("zhulang.com") > -1)
                return false;

            if (i_url.ToLower().IndexOf("rongshuxia.com") > -1)
                return false;

            if (i_url.ToLower().IndexOf("17k.com") > -1)
                return false;

            if (i_url.ToLower().IndexOf("lcread.com") > -1)
                return false;

            if (i_url.ToLower().IndexOf("book.qq.com") > -1)
                return false;

            return true;
        }

    }
}
